import pandas as pd
from python_ai.common.xcommon import sep

pd.set_option('display.max_rows', None, 'display.max_columns', None, 'display.max_colwidth', 1000, 'display.expand_frame_repr', False)

# ATTENTION
# sep: str, default ‘,’
# Delimiter to use. If sep is None, the C engine cannot automatically detect the separator,
# but the Python parsing engine can, meaning the latter will be used and automatically detect the separator by
# Python’s builtin sniffer tool, csv.Sniffer. In addition, separators longer than 1 character and different from
# '\s+' will be interpreted as regular expressions and will also force the use of the Python parsing engine. Note
# that regex delimiters are prone to ignoring quoted data. Regex example: '\r\t'.

df = pd.read_csv(
    r'..\..\..\..\..\large_data\ML2\stopwords1.txt',
    # r'..\..\..\..\..\large_data\ML2\passwd',
    header=None,

    # ATTENTION
    sep=r'\n',  # for file with only one column, any token (len >= 2) works unless it is contained by some value.
    # pandas.errors.ParserError: Expected 1 fields in line 17, saw 2. Error could possibly be due to quotes being ignored when a multi-char delimiter is used.

    # sep=r':\d+:',
    engine='python',
    encoding='utf8',
)
print(df)
